from typing import Pattern

from bs4 import BeautifulSoup
import re
import urllib.request,urllib.error
import xlwt
import sqlite3
import random
import time
from urllib.parse import unquote,urlencode
from datetime import datetime






def main():
    taskMap={
        #"computer":['cpu','主板','散热','显卡','电源','机箱','内存','硬盘'],
        
        # "computer":['cpu intel','cpu AMD',
        #             '主板 ASUS','主板 微星','主板 技嘉','主板 铭瑄',
        #             '散热 利民','散热 瓦尔基里','散热 雅浚', '散热 九州风神','散热 乔思伯',
        #             '显卡 七彩虹','显卡 ASUS','显卡 微星','显卡 技嘉','显卡 影驰','显卡 索泰','显卡 蓝宝石','显卡 盈通','显卡 耕升','显卡 铭瑄',
        #             '内存 金士顿','内存 金百达','内存 光威','内存 三星','内存 美商海盗船','内存 威刚','内存 芝奇','内存 宏碁','内存 阿斯加特','内存 雷克沙',
        #             '固态 三星','固态 梵想','固态 致钛','固态 WD','固态 金士顿','固态 宏碁掠夺者','固态 光威','固态 铠侠',
        #             '电源 长城','电源 航嘉','电源 华硕','电源 鑫谷','电源 海韵','电源 酷冷至尊','电源 全汉','电源 艾湃电竞',
        #             '机箱 华硕','机箱 先马','机箱 航嘉','机箱 爱国者','机箱 长城','机箱 玩嘉','机箱 乔思伯','机箱 TT','机箱 酷冷至尊','机箱 鑫谷','机箱 追风者','机箱 大水牛','机箱 安钛克',
        #             '机箱 金河田','机箱 半岛铁盒',
        #             '风扇 利民','风扇 棱镜',
        #             '显示器 AOC','显示器 LG','显示器 泰坦军团',
        #             ],
        
        #"华硕主板": ["华硕 主板"],
        #"硬盘": ["硬盘"],
        #"内存": ["内存"],
        #"金百达": ["金百达"],
        #"内存32G": ["内存 32G"],
        #"内存宏碁": ["内存 宏碁"],
        #"显卡40系": ["RTX 4060","RTX 4060 TI","RTX 4070 TI","RTX 4070 Super","RTX 4070 Super","RTX 4080","RTX 4080 Super","RTX 4090D"],

        # "显卡AMD": ["显卡 AMD"],
        # "主板Z790": ["主板Z790 ","华硕 Z790" ,"微星 Z790","技嘉 Z790"],
        # "主板B760": ["主板B760 ","华硕 B760" ,"微星 B760","技嘉 B760"],
        # "板U套装": ["I3 12100F ","I5 12400F" ,"I5 12600KF","I5 13600KF","I5 14600KF","I7 13700KF","I7 14700KF","I9 14900KF"],
        # "鼠标键盘耳机": ["鼠标 ","键盘" ,"耳机"],
        #"主板H610": ["主板 H610"],
        #"机箱海景房": ["机箱 海景房"],
        #"CPUAMD": ["CPU R5 5500","CPU R5 5600G","CPU R5 5600","CPU R7 5700X3D","CPU R7 5800X3D","CPU R5 7500F","CPU R7 7700","CPU R9 7900X","CPU R9 7950X"],
        # "主机价格":["1000元主机","2000元主机","3000元主机","4000元主机","5000元主机","6000元主机","7000元主机","8000元主机","9000元主机","10000元主机",
        #          "11000元主机","12000元主机","13000元主机","14000元主机","15000元主机","16000元主机","17000元主机","18000元主机","19000元主机","200000元主机",
        #          "21000元主机","22000元主机","23000元主机","24000元主机","25000元主机","26000元主机","27000元主机","28000元主机","29000元主机","300000元主机",
        #          ],
        #"主机深度学习": ["主机 深度学习"],
        #"主机游戏": ["主机 游戏"],
        #"主机建模渲染": ["主机 建模 渲染"],
        #"主机直播": ["主机 直播"],
        #"主机编程": ["主机 编程"],
        #"主机视频剪辑": ["主机 视频剪辑"],
        #"显卡40系七彩虹": ["RTX 4060 七彩虹 ","RTX 4060 TI 七彩虹","RTX 4070 七彩虹","RTX 4070 TI 七彩虹","RTX 4070 Super","RTX 4070 TI SUPER 七彩虹","RTX 4080 七彩虹 ","RTX 4080 Super 七彩虹","RTX 4090D 七彩虹"],
        #"显卡40系华硕": ["RTX 4060 华硕 ","RTX 4060 TI 华硕","RTX 4070 华硕","RTX 4070 TI 华硕","RTX 4070 Super","RTX 4070 TI SUPER 华硕","RTX 4080 华硕 ","RTX 4080 Super 华硕","RTX 4090D 华硕"],
        #"显卡40系微星": ["RTX 4060 微星 ","RTX 4060 TI 微星","RTX 4070 微星","RTX 4070 TI 微星","RTX 4070 Super","RTX 4070 TI SUPER 微星","RTX 4080 微星 ","RTX 4080 Super 微星","RTX 4090D 微星"],
        "显卡40系技嘉": ["RTX 4060 技嘉 ","RTX 4060 TI 技嘉","RTX 4070 技嘉","RTX 4070 TI 技嘉","RTX 4070 Super","RTX 4070 TI SUPER 技嘉","RTX 4080 技嘉 ","RTX 4080 Super 技嘉","RTX 4090D 技嘉"],
    }
    # 获取当前时间
    now = datetime.now()
    # 格式化时间字符串，不含空格
    formatted_time = now.strftime('%Y-%m-%d_%H-%M')
    for key in taskMap:
        keywordlist1 = taskMap[key]
        datalist = []
        for element in keywordlist1:
            keyword=element
            keywordencode = urlencode({'keyword':keyword})
            print("搜索关键词:"+keywordencode)
            baseurl = "https://search.jd.com/Search?"+keywordencode+"&pvid=20b577a9a2d1457bb5abf1aebec16ce7&page="
            savepath=".\\store\\"+key+formatted_time+".xls"
            #爬取网页 并提取信息
            data=[">>>>>"+keyword+"<<<<<<<","<<<<<","<<<<<<","<<<<<"]
            datalist.append(data)
            getData(datalist,baseurl)
    
        #保存数据
        saveData(datalist,savepath)



url1="&s=1&click=0"#补充baseurl形成网页列表形式
# 电脑信息查找规则
findName = re.compile(r'<em>(.*?)</em>', re.DOTALL)    #生成正则表达式的对象，表示规则 re.DOTALL忽略换行符
pattern = re.compile(r'<[^>]+>',re.S) #去除各种<>
price = re.compile(r'<i data-price="(.*?)</i>')
shop = re.compile(r'<a class="curr-shop hd-shopname"(.*?)</a>')
web = re.compile(r'<a href="(.*?)" onclick="')

def remove_html_tags(html):
    result = pattern.sub('', html)
    if len(result) == 1:
        print("html empty:"+html)
    return result

def remove_list_tags_from_list(lst):
    return [remove_html_tags(item) for item in lst]

def getData(datalist,baseurl):
    for i in range(0,3):
        #print("hahahahahah")
        sleep_time = random.randint(2, 10)  # 生成一个1到5秒之间的随机整数
        #print(f"程序将暂停 {sleep_time} 秒...")
        time.sleep(sleep_time)

        url = baseurl+str(i+1)+url1
        html=askURL(url)
        print(url)
        
        #逐一进行解析
        soup =BeautifulSoup(html,"html.parser")
        for item in soup.select("div.gl-i-wrap"):
            #print(item)
            data = []#保存一部电脑的全部信息
            item = str(item)
            print("===item======"+item+"=========")
            name = re.findall(findName,item)#re库通过正则表达式查找指定的字符串

            if name.__len__()!=0: #不抓取带有 京东电脑 标签的电脑
                print("///////name begin////////")
                print(name)
                print("///////name end////////")
                clearName = remove_list_tags_from_list(name)
                print("///////clearName begin////////")
                print(clearName)
                print("///////clearName end////////")
                data.append(clearName)#在data中加入clearName

                Price=re.findall(price,item)#抓取价格标签内数据
                index=Price[0].find('>')#只提取价格标签数字部分
                #print(Price[0][index+1:])
                data.append(Price[0][index+1:])

                Web = re.findall(web,item)#抓取网页部分信息
                #print("https:"+Web[0])
                data.append("https:"+Web[0])

                Shop=re.findall(shop,item)#商铺名称
                if len(Shop) > 0:
                    #print("shop:"+Shop[0])
                    start_index = Shop[0].find(">") + 1
                    #print(Shop[0][start_index:])  # 输出 "World!"
                    data.append(Shop[0][start_index:])
                else:
                    data.append("")

                datalist.append(data)
        for i in range(4):
            data = ["","","",""]
            datalist.append(data)

                    
    #return datalist

#得到指定一个url的网页信息
def askURL(url):
    #用户代理表示不是爬虫，模拟头部信息
    
    head={"authority":"search.jd.com",
"method":"GET",
"path":"/Search?keyword=%E6%98%BE%E5%8D%A1&enc=utf-8&wq=%E6%98%BE%E5%8D%A1&pvid=19cc324c919d4782b1d45a695e91558c",
"scheme":"https",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language":"zh-CN,zh;q=0.9",
"Cookie":"shshshfpa=38695893-467c-2f0b-3d96-5974ea46d432-1705406011; shshshfpx=38695893-467c-2f0b-3d96-5974ea46d432-1705406011; TrackID=1f_mAfytScnx_b4MDsLH7sa5AtOfGU9IXC2GD6bC1XK17rbq2KCQG5fphWxT_s5DFWDVnqIWtZiGLw_41inGx5BaNqPPE5Mw8Fl5PVHJHIb4; qrsc=3; __jdu=17114359180581914754099; areaId=2; ipLoc-djd=2-2830-51803-0; unpl=JF8EAMpnNSttCxlcBxMHHhAVTVpTW11cG0RXPGFSBl5bGwACGwoSRhF7XlVdWBRKFR9sbhRXXVNOVg4YBCsSEXteXVdZDEsWC2tXVgQFDQ8VXURJQlZAFDNVCV9dSRZRZjJWBFtdT1xWSAYYRRMfDlAKDlhCR1FpMjVkXlh7VAQrAh0SEE5fUFZdAUoTCm9vBVFbXE5VDRMyGiIXe21kW1UMSxQLX2Y1VW0aHwgGHgMYEBMGXVJeXQ1JEwtvbgRQVFhDVAAdBh4TGENtVW5e; PCSYCityID=CN_310000_310100_0; shshshfpb=BApXeirJFhutAn_aDtDvBv49fMANRgGXoBkcBZz5p9xJ1MjNLH4C2; thor=BB415EE7BFF3D18698B502A2D8A96EEC6FECD5CD114C36BBB4E537E50C2742962D58C9C38CD1EB8AFAE3B0062A082C59D909342A8C34DDBA0860FC1DC5CE70AEDA851040B59B6AB23ECF36D5FCEA7D5C0D4B0939E8D53944A16890193804F2F9CAF2BD9435A90D316A19C04ED2B4D03F2AB327F7BBC7ECBCC1294E9EA4F62FD7874F274E2C1CC4EDBB417897BD3644D6; flash=2_R_ErKMS_Z7S-tdjd4YpK84YTAWB5t06gmdhdQZ0x7vIK3jOr2rYdMg_6qKwwTGXYxDfHLd1FKq8uFVZLS27GEaIE2sieUsgNRxy4D1En4-s*; pinId=kz2UyqNKhd_-lkf6wCUe3Q; pin=13817246127_p; unick=jd_%E7%A2%A7%E5%A4%A9%E4%BA%91%E6%9C%88; ceshi3.com=000; _tp=A1hNrcNaPsCDNethPvI0yQ%3D%3D; _pst=13817246127_p; 3AB9D23F7A4B3C9B=T6OKMLCBDUKQEACQXT4LIY2SQBYDCTXA7QBN6G5YI46TUAGSE6KIBUOG2DLZGKLVTMQL5ISRNMGAVJK3TWHIIBNDTA; __jda=76161171.17114359180581914754099.1711435918.1711446744.1711633406.3; __jdc=76161171; __jdb=76161171.4.17114359180581914754099|3.1711633406; __jdv=76161171|baidu-pinzhuan|t_288551095_baidupinzhuan|cpc|0f3d30c8dba7459bb52f2eb5eba8ac7d_0_bc939443476641eabeb7fc22ae6198e0|1711633551394; 3AB9D23F7A4B3CSS=jdd03T6OKMLCBDUKQEACQXT4LIY2SQBYDCTXA7QBN6G5YI46TUAGSE6KIBUOG2DLZGKLVTMQL5ISRNMGAVJK3TWHIIBNDTAAAAAMOQVHVQSYAAAAADZC5TCFV3P67S4X; _gia_d=1",
"Sec-Ch-Ua":"\"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"",
"Sec-Ch-Ua-Mobile":"?0",
"Sec-Ch-Ua-Platform":"\"Windows\"",
"Sec-Fetch-Dest":"document",
"Sec-Fetch-Mode":"navigate",
"Sec-Fetch-Site":"same-site",
"Sec-Fetch-User":"?1",
"Upgrade-Insecure-Requests":"1",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
 }
    request=urllib.request.Request(url,headers=head)
    html=""
    try:
        response=urllib.request.urlopen(request)
        html=response.read().decode("utf-8")
        #print(html)
    #异常处理机制
    except urllib.error.URLError as e:
        if(hasattr(e,"code")):
            print(e,"code")
        if(hasattr(e,"reason")):
            print(e,"reason")
    return html

def saveData(datalist,savepath):
    book = xlwt.Workbook(encoding="utf-8")#utf-8编码方式
    sheet = book.add_sheet('information')#命名Excel文件sheet名字为information
    col = ("标签","价格","网址","品牌")
    for i in range(0,4):
        sheet.write(0,i,col[i])#列名的写入
    for i in range(0,len(datalist)):#
        #print("第%d条"%i)#将抓取的前1000条数据存入Excel中
        data = datalist[i]#按行提取数据
        for j in range(0,4):
            sheet.write(i+1,j,data[j])#写入数据
    book.save(savepath)#保存Excel路径

if __name__=="__main__":
    main()